package taskspider.retrieval.wordnet;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;

import taskspider.util.properties.*;

/**
 * Convert the prolog file wn_s.pl from the <a href="http://www.cogsci.princeton.edu/2.0/WNprolog-2.0.tar.gz">WordNet prolog download</a>
 * into a Lucene index suitable for looking up synonyms and performing query expansion ({@link SynExpand#expand SynExpand.expand(...)}).
 *
 * This has been tested with WordNet 2.0.
 *
 * The index has fields named "word" ({@link #F_WORD})
 * and "syn" ({@link #F_SYN}).
 * <p>
 * The source word (such as 'big') can be looked up in the
 * "word" field, and if present there will be fields named "syn"
 * for every synonym. What's tricky here is that there could be <b>multiple</b>
 * fields with the same name, in the general case for words that have multiple synonyms.
 * That's not a problem with Lucene, you just use {@link org.apache.lucene.document.Document#getValues}
 * </p>
 * <p>
 * While the WordNet file distinguishes groups of synonyms with
 * related meanings we don't do that here.
 * </p>
 *
 * This can take 4 minutes to execute and build an index on a "fast" system and the index takes up almost 3 MB.
 *
 * @author Dave Spencer, dave&#064;searchmorph.com
 * @see <a href="http://www.cogsci.princeton.edu/~wn/">WordNet home page</a>
 * @see <a href="http://www.cogsci.princeton.edu/~wn/man/prologdb.5WN.html">prologdb man page</a>
 * @see <a href="http://www.hostmon.com/rfc/advanced.jsp">sample site that uses it</a>
 */
public class Syns2Index
{
	/**
	 *
	 */
	private static final PrintStream o = System.out;

	/**
	 *
	 */
	private static final PrintStream err = System.err;
	
	/**
	 *
	 */
	public static final String F_SYN = "syn";

	/**
	 *
	 */
	public static final String F_WORD = "word";

	/**
	 *
	 */
    private static final Analyzer ana = new StandardAnalyzer();
    
    
    public static void generateIndex() {
    	try {
			// get command line arguments
			String prologFilename = PropertiesReader.getProperty("prologFilename"); // name of file "wn_s.pl"
			String indexDir = PropertiesReader.getProperty("wordnetIndexPath");

			// ensure that the prolog file is readable
			if (! (new File(prologFilename)).canRead())
			{
				err.println("Error: cannot read Prolog file: " + prologFilename);
				System.exit(1);
			}
			// exit if the target index directory already exists
			if ((new File(indexDir)).isDirectory())
			{
				err.println("Error: index directory already exists: " + indexDir);
				err.println("Please specify a name of a non-existent directory");
				System.exit(1);
			}

			o.println("Opening Prolog file " + prologFilename);
			final FileInputStream fis = new FileInputStream(prologFilename);
			final BufferedReader br = new BufferedReader(new InputStreamReader(fis));
			String line;

			// maps a word to all the "groups" it's in
			final Map word2Nums = new TreeMap();
			// maps a group to all the words in it
			final Map num2Words = new TreeMap();
			// number of rejected words
			int ndecent = 0;

			// status output
			int mod = 1;
			int row = 1;
			// parse prolog file
			o.println( "[1/2] Parsing " + prologFilename);
			while ((line = br.readLine()) != null)
			{
				// occasional progress
				if ((++row) % mod == 0) // periodically print out line we read in
				{
					mod *= 2;
					o.println("\t" + row + " " + line + " " + word2Nums.size()
							+ " " + num2Words.size() + " ndecent=" + ndecent);
				}

				// syntax check
				if (! line.startsWith("s("))
				{
					err.println("OUCH: " + line);
					System.exit(1);
				}

				// parse line
				line = line.substring(2);
				int comma = line.indexOf(',');
				String num = line.substring(0, comma);
				int q1 = line.indexOf('\'');
				line = line.substring(q1 + 1);
				int q2 = line.indexOf('\'');
				String word = line.substring(0, q2).toLowerCase();

				// make sure is a normal word
				if (! isDecent(word))
				{
					ndecent++;
					continue; // don't store words w/ spaces
				}

				// 1/2: word2Nums map
				// append to entry or add new one
				List lis =(List) word2Nums.get(word);
				if (lis == null)
				{
					lis = new LinkedList();
					lis.add(num);
					word2Nums.put(word, lis);
				}
				else
					lis.add(num);

				// 2/2: num2Words map
				lis = (List) num2Words.get(num);
				if (lis == null)
				{
					lis = new LinkedList();
					lis.add(word);
					num2Words.put(num, lis);
				}
				else
					lis.add(word);
			}

			// close the streams
			fis.close();
			br.close();

			// create the index
			o.println( "[2/2] Building index to store synonyms, " +
					" map sizes are " + word2Nums.size() + " and " + num2Words.size());
			index(indexDir, word2Nums, num2Words);
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (Throwable e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
    }

    /**
     * Checks to see if a word contains only alphabetic characters by
     * checking it one character at a time.
     *
     * @param s string to check
     * @return <code>true</code> if the string is decent
     */
    private static boolean isDecent(String s)
    {
        int len = s.length();
        for (int i = 0; i < len; i++)
        {
            if (!Character.isLetter(s.charAt(i)))
            {
                return false;
            }
        }
        return true;
    }

    /**
     * Forms a Lucene index based on the 2 maps.
     *
     * @param indexDir the direcotry where the index should be created
     * @param word2Nums
     * @param num2Words
     */
    private static void index(String indexDir, Map word2Nums, Map num2Words)
        throws Throwable
    {
        int row = 0;
        int mod = 1;

        // override the specific index if it already exists
        IndexWriter writer = new IndexWriter(indexDir, ana, true);
        writer.setUseCompoundFile(true); // why?
		// blindly up these parameters for speed
		writer.setMergeFactor( writer.getMergeFactor() * 2);
		writer.setMaxBufferedDocs( writer.getMaxBufferedDocs() * 2);
        Iterator i1 = word2Nums.keySet().iterator();
        while (i1.hasNext()) // for each word
        {
            String g = (String) i1.next();
            Document doc = new Document();

            int n = index(word2Nums, num2Words, g, doc);
            if (n > 0)
            {
				doc.add( new Field( F_WORD, g, Field.Store.YES, Field.Index.UN_TOKENIZED));
                if ((++row % mod) == 0)
                {
                    o.println("\trow=" + row + "/" + word2Nums.size() + " doc= " + doc);
                    mod *= 2;
                }
                writer.addDocument(doc);
            } // else degenerate
        }
		o.println( "Optimizing..");
        writer.optimize();
        writer.close();
    }

    /**
     * Given the 2 maps fills a document for 1 word.
     */
    private static int index(Map word2Nums, Map num2Words, String g, Document doc)
        throws Throwable
    {
        List keys = (List) word2Nums.get(g); // get list of key#'s
        Iterator i2 = keys.iterator();

        Set already = new TreeSet(); // keep them sorted

        // pass 1: fill up 'already' with all words
        while (i2.hasNext()) // for each key#
        {
            already.addAll((List) num2Words.get(i2.next())); // get list of words
        }
        int num = 0;
        already.remove(g); // of course a word is it's own syn
        Iterator it = already.iterator();
        while (it.hasNext())
        {
            String cur = (String) it.next();
            // don't store things like 'pit bull' -> 'american pit bull'
            if (!isDecent(cur))
            {
                continue;
            }
            num++;
			doc.add( new Field( F_SYN, cur, Field.Store.YES, Field.Index.NO));
        }
        return num;
    }

	/**
	 *
	 */
    private static void usage()
    {
        o.println("\n\n" +
            "java org.apache.lucene.wordnet.Syns2Index <prolog file> <index dir>\n\n");
    }

}
